{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:26.906284Z",
     "start_time": "2020-04-09T07:23:25.377360Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "conv2d (Conv2D)              (None, 56, 56, 96)        34944     \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 56, 56, 96)        384       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 27, 27, 96)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 27, 27, 256)       614656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 27, 27, 256)       1024      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 13, 13, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 13, 13, 384)       885120    \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 13, 13, 384)       1327488   \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 13, 13, 256)       884992    \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 13, 13, 256)       1024      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 9216)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 4096)              37752832  \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 62,380,776\n",
      "Trainable params: 62,379,560\n",
      "Non-trainable params: 1,216\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Input, Conv2D, BatchNormalization, MaxPool2D, Flatten, Dense, Dropout\n",
    "\n",
    "input = Input(shape=(224, 224, 3))\n",
    "\n",
    "x = Conv2D(filters=96,\n",
    "           kernel_size=11,\n",
    "           strides=4,\n",
    "           padding='same',\n",
    "           activation='relu')(input)  # 1st convolutional layer\n",
    "x = BatchNormalization()(x)\n",
    "x = MaxPool2D(pool_size=3, strides=2)(x)\n",
    "\n",
    "x = Conv2D(filters=256, kernel_size=5, padding='same',\n",
    "           activation='relu')(x)  # 2nd convolutional layer\n",
    "x = BatchNormalization()(x)\n",
    "x = MaxPool2D(pool_size=3, strides=2)(x)\n",
    "\n",
    "x = Conv2D(filters=384, kernel_size=3, padding='same',\n",
    "           activation='relu')(x)  # 3rd convolutional layer\n",
    "\n",
    "x = Conv2D(filters=384, kernel_size=3, padding='same',\n",
    "           activation='relu')(x)  # 4th convolutional layer\n",
    "\n",
    "x = Conv2D(filters=256, kernel_size=3, padding='same',\n",
    "           activation='relu')(x)  # 5th convolutional layer\n",
    "x = BatchNormalization()(x)\n",
    "x = MaxPool2D(pool_size=3, strides=2)(x)\n",
    "\n",
    "x = Flatten()(x)\n",
    "x = Dense(units=4096, activation='relu')(x)\n",
    "x = Dense(units=4096, activation='relu')(x)\n",
    "x = Dropout(rate=0.5)(x)\n",
    "\n",
    "output = Dense(units=1000, activation='softmax')(x)\n",
    "\n",
    "from tensorflow.keras import Model\n",
    "model = Model(inputs=input, outputs=output)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:27.048505Z",
     "start_time": "2020-04-09T07:23:26.907225Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-a24ca5fc5d11>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muniform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tf' is not defined"
     ]
    }
   ],
   "source": [
    "x = tf.random.uniform(shape=(1, 224, 224, 3))\n",
    "model(x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VGGNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:27.049993Z",
     "start_time": "2020-04-09T07:23:25.721Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Conv2D, MaxPool2D, Flatten, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:27.050554Z",
     "start_time": "2020-04-09T07:23:25.733Z"
    }
   },
   "outputs": [],
   "source": [
    "input = Input(shape=(224, 224, 3))\n",
    "\n",
    "x = Conv2D(filters=64, kernel_size=3, padding='same', activation='relu')(input)\n",
    "x = Conv2D(filters=64, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPool2D(pool_size=2, strides=2, padding='same')(x)\n",
    "\n",
    "x = Conv2D(filters=128, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = Conv2D(filters=128, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPool2D(pool_size=2, strides=2, padding='same')(x)\n",
    "\n",
    "x = Conv2D(filters=256, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = Conv2D(filters=256, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPool2D(pool_size=2, strides=2, padding='same')(x)\n",
    "\n",
    "x = Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPool2D(pool_size=2, strides=2, padding='same')(x)\n",
    "\n",
    "x = Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = Conv2D(filters=512, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPool2D(pool_size=2, strides=2, padding='same')(x)\n",
    "\n",
    "x = Flatten()(x)\n",
    "x = Dense(units=4096, activation='relu')(x)\n",
    "x = Dense(units=4096, activation='relu')(x)\n",
    "output = Dense(units=1000, activation='softmax')(x)\n",
    "\n",
    "from tensorflow.keras import Model\n",
    "model = Model(inputs=input, outputs=output)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:27.051082Z",
     "start_time": "2020-04-09T07:23:25.739Z"
    }
   },
   "outputs": [],
   "source": [
    "x = tf.random.uniform(shape=(1, 224, 224, 3))\n",
    "model(x).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GoogleNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:27.051649Z",
     "start_time": "2020-04-09T07:23:25.752Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Conv2D, MaxPool2D, Concatenate, AvgPool2D, Dropout, Flatten, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:27.052246Z",
     "start_time": "2020-04-09T07:23:25.760Z"
    }
   },
   "outputs": [],
   "source": [
    "def inception_block(x, filters):\n",
    "    t1 = Conv2D(filters=filters[0], kernel_size=1, activation='relu')(x)\n",
    "\n",
    "    t2 = Conv2D(filters=filters[1], kernel_size=1, activation='relu')(x)\n",
    "    t2 = Conv2D(filters=filters[2],\n",
    "                kernel_size=3,\n",
    "                padding='same',\n",
    "                activation='relu')(t2)\n",
    "\n",
    "    t3 = Conv2D(filters=filters[3], kernel_size=1, activation='relu')(x)\n",
    "    t3 = Conv2D(filters=filters[4],\n",
    "                kernel_size=5,\n",
    "                padding='same',\n",
    "                activation='relu')(t3)\n",
    "\n",
    "    t4 = MaxPool2D(pool_size=3, strides=1, padding='same')(x)\n",
    "    t4 = Conv2D(filters=filters[5], kernel_size=1, activation='relu')(t4)\n",
    "\n",
    "    output = Concatenate()([t1, t2, t3, t4])\n",
    "    return output\n",
    "\n",
    "\n",
    "input = Input(shape=(224, 224, 3))\n",
    "x = Conv2D(filters=64,\n",
    "           kernel_size=7,\n",
    "           strides=2,\n",
    "           padding='same',\n",
    "           activation='relu')(input)\n",
    "x = MaxPool2D(pool_size=3, strides=2, padding='same')(x)\n",
    "\n",
    "x = Conv2D(filters=64, kernel_size=1, activation='relu')(x)\n",
    "x = Conv2D(filters=192, kernel_size=3, padding='same', activation='relu')(x)\n",
    "x = MaxPool2D(pool_size=3, strides=2)(x)\n",
    "\n",
    "x = inception_block(x, filters=[64, 96, 128, 16, 32, 32])\n",
    "x = inception_block(x, filters=[128, 128, 192, 32, 96, 64])\n",
    "x = MaxPool2D(pool_size=3, strides=2, padding='same')(x)\n",
    "\n",
    "x = inception_block(x, filters=[192, 96, 208, 16, 48, 64])\n",
    "x = inception_block(x, filters=[160, 112, 224, 24, 64, 64])\n",
    "x = inception_block(x, filters=[128, 128, 256, 24, 64, 64])\n",
    "x = inception_block(x, filters=[112, 144, 288, 32, 64, 64])\n",
    "x = inception_block(x, filters=[256, 160, 320, 32, 128, 128])\n",
    "x = MaxPool2D(pool_size=3, strides=2, padding='same')(x)\n",
    "\n",
    "x = inception_block(x, filters=[256, 160, 320, 32, 128, 128])\n",
    "x = inception_block(x, filters=[384, 192, 384, 48, 128, 128])\n",
    "x = AvgPool2D(pool_size=7, strides=1)(x)\n",
    "x = Dropout(rate=0.4)(x)\n",
    "\n",
    "x = Flatten()(x)\n",
    "output = Dense(units=1000, activation='softmax')(x)\n",
    "\n",
    "from tensorflow.keras import Model\n",
    "\n",
    "model = Model(inputs=input, outputs=output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:23:27.052752Z",
     "start_time": "2020-04-09T07:23:25.768Z"
    }
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MobileNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:51:15.783357Z",
     "start_time": "2020-04-09T07:51:14.902613Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, DepthwiseConv2D, \\\n",
    "     Conv2D, BatchNormalization, ReLU, AvgPool2D, Flatten, Dense\n",
    "\n",
    "\n",
    "def mobilenet_block(x, filters, strides):\n",
    "    x = DepthwiseConv2D(kernel_size=3, strides=strides, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU()(x)\n",
    "\n",
    "    x = Conv2D(filters=filters, kernel_size=1, strides=1, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "INPUT_SHAPE = 224, 224, 3\n",
    "\n",
    "input = Input(INPUT_SHAPE)\n",
    "x = Conv2D(filters=32, kernel_size=3, strides=2, padding='same')(input)\n",
    "x = BatchNormalization()(x)\n",
    "x = ReLU()(x)\n",
    "\n",
    "x = mobilenet_block(x, filters=64, strides=1)\n",
    "\n",
    "x = mobilenet_block(x, filters=128, strides=2)\n",
    "x = mobilenet_block(x, filters=128, strides=1)\n",
    "\n",
    "x = mobilenet_block(x, filters=256, strides=2)\n",
    "x = mobilenet_block(x, filters=256, strides=1)\n",
    "\n",
    "x = mobilenet_block(x, filters=512, strides=2)\n",
    "for _ in range(5):\n",
    "    x = mobilenet_block(x, filters=512, strides=1)\n",
    "\n",
    "x = mobilenet_block(x, filters=1024, strides=2)\n",
    "x = mobilenet_block(x, filters=1024, strides=1)\n",
    "\n",
    "x = AvgPool2D(pool_size=7, strides=1)(x)\n",
    "output = Dense(units=1000, activation='softmax')(x)\n",
    "\n",
    "from tensorflow.keras import Model\n",
    "\n",
    "model = Model(inputs=input, outputs=output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:51:25.257324Z",
     "start_time": "2020-04-09T07:51:25.238770Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 112, 112, 32)      896       \n",
      "_________________________________________________________________\n",
      "batch_normalization_27 (Batc (None, 112, 112, 32)      128       \n",
      "_________________________________________________________________\n",
      "re_lu_27 (ReLU)              (None, 112, 112, 32)      0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_13 (Depthwi (None, 112, 112, 32)      320       \n",
      "_________________________________________________________________\n",
      "batch_normalization_28 (Batc (None, 112, 112, 32)      128       \n",
      "_________________________________________________________________\n",
      "re_lu_28 (ReLU)              (None, 112, 112, 32)      0         \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 112, 112, 64)      2112      \n",
      "_________________________________________________________________\n",
      "batch_normalization_29 (Batc (None, 112, 112, 64)      256       \n",
      "_________________________________________________________________\n",
      "re_lu_29 (ReLU)              (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_14 (Depthwi (None, 56, 56, 64)        640       \n",
      "_________________________________________________________________\n",
      "batch_normalization_30 (Batc (None, 56, 56, 64)        256       \n",
      "_________________________________________________________________\n",
      "re_lu_30 (ReLU)              (None, 56, 56, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_16 (Conv2D)           (None, 56, 56, 128)       8320      \n",
      "_________________________________________________________________\n",
      "batch_normalization_31 (Batc (None, 56, 56, 128)       512       \n",
      "_________________________________________________________________\n",
      "re_lu_31 (ReLU)              (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_15 (Depthwi (None, 56, 56, 128)       1280      \n",
      "_________________________________________________________________\n",
      "batch_normalization_32 (Batc (None, 56, 56, 128)       512       \n",
      "_________________________________________________________________\n",
      "re_lu_32 (ReLU)              (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_17 (Conv2D)           (None, 56, 56, 128)       16512     \n",
      "_________________________________________________________________\n",
      "batch_normalization_33 (Batc (None, 56, 56, 128)       512       \n",
      "_________________________________________________________________\n",
      "re_lu_33 (ReLU)              (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_16 (Depthwi (None, 28, 28, 128)       1280      \n",
      "_________________________________________________________________\n",
      "batch_normalization_34 (Batc (None, 28, 28, 128)       512       \n",
      "_________________________________________________________________\n",
      "re_lu_34 (ReLU)              (None, 28, 28, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_18 (Conv2D)           (None, 28, 28, 256)       33024     \n",
      "_________________________________________________________________\n",
      "batch_normalization_35 (Batc (None, 28, 28, 256)       1024      \n",
      "_________________________________________________________________\n",
      "re_lu_35 (ReLU)              (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_17 (Depthwi (None, 28, 28, 256)       2560      \n",
      "_________________________________________________________________\n",
      "batch_normalization_36 (Batc (None, 28, 28, 256)       1024      \n",
      "_________________________________________________________________\n",
      "re_lu_36 (ReLU)              (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_19 (Conv2D)           (None, 28, 28, 256)       65792     \n",
      "_________________________________________________________________\n",
      "batch_normalization_37 (Batc (None, 28, 28, 256)       1024      \n",
      "_________________________________________________________________\n",
      "re_lu_37 (ReLU)              (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_18 (Depthwi (None, 14, 14, 256)       2560      \n",
      "_________________________________________________________________\n",
      "batch_normalization_38 (Batc (None, 14, 14, 256)       1024      \n",
      "_________________________________________________________________\n",
      "re_lu_38 (ReLU)              (None, 14, 14, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_20 (Conv2D)           (None, 14, 14, 512)       131584    \n",
      "_________________________________________________________________\n",
      "batch_normalization_39 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_39 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_19 (Depthwi (None, 14, 14, 512)       5120      \n",
      "_________________________________________________________________\n",
      "batch_normalization_40 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_40 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_21 (Conv2D)           (None, 14, 14, 512)       262656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_41 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_41 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_20 (Depthwi (None, 14, 14, 512)       5120      \n",
      "_________________________________________________________________\n",
      "batch_normalization_42 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_42 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_22 (Conv2D)           (None, 14, 14, 512)       262656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_43 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_43 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_21 (Depthwi (None, 14, 14, 512)       5120      \n",
      "_________________________________________________________________\n",
      "batch_normalization_44 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_44 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_23 (Conv2D)           (None, 14, 14, 512)       262656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_45 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_45 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_22 (Depthwi (None, 14, 14, 512)       5120      \n",
      "_________________________________________________________________\n",
      "batch_normalization_46 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_46 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_24 (Conv2D)           (None, 14, 14, 512)       262656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_47 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_47 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_23 (Depthwi (None, 14, 14, 512)       5120      \n",
      "_________________________________________________________________\n",
      "batch_normalization_48 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_48 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_25 (Conv2D)           (None, 14, 14, 512)       262656    \n",
      "_________________________________________________________________\n",
      "batch_normalization_49 (Batc (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "re_lu_49 (ReLU)              (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_24 (Depthwi (None, 7, 7, 512)         5120      \n",
      "_________________________________________________________________\n",
      "batch_normalization_50 (Batc (None, 7, 7, 512)         2048      \n",
      "_________________________________________________________________\n",
      "re_lu_50 (ReLU)              (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_26 (Conv2D)           (None, 7, 7, 1024)        525312    \n",
      "_________________________________________________________________\n",
      "batch_normalization_51 (Batc (None, 7, 7, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "re_lu_51 (ReLU)              (None, 7, 7, 1024)        0         \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_25 (Depthwi (None, 7, 7, 1024)        10240     \n",
      "_________________________________________________________________\n",
      "batch_normalization_52 (Batc (None, 7, 7, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "re_lu_52 (ReLU)              (None, 7, 7, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_27 (Conv2D)           (None, 7, 7, 1024)        1049600   \n",
      "_________________________________________________________________\n",
      "batch_normalization_53 (Batc (None, 7, 7, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "re_lu_53 (ReLU)              (None, 7, 7, 1024)        0         \n",
      "_________________________________________________________________\n",
      "average_pooling2d_1 (Average (None, 1, 1, 1024)        0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1, 1, 1000)        1025000   \n",
      "=================================================================\n",
      "Total params: 4,264,808\n",
      "Trainable params: 4,242,920\n",
      "Non-trainable params: 21,888\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T07:59:36.917260Z",
     "start_time": "2020-04-09T07:59:35.630856Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_89 (Conv2D)              (None, 112, 112, 64) 9472        input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_102 (BatchN (None, 112, 112, 64) 256         conv2d_89[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_109 (ReLU)                (None, 112, 112, 64) 0           batch_normalization_102[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 56, 56, 64)   0           re_lu_109[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_90 (Conv2D)              (None, 56, 56, 64)   4160        max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_103 (BatchN (None, 56, 56, 64)   256         conv2d_90[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_110 (ReLU)                (None, 56, 56, 64)   0           batch_normalization_103[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_91 (Conv2D)              (None, 56, 56, 64)   36928       re_lu_110[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_104 (BatchN (None, 56, 56, 64)   256         conv2d_91[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_111 (ReLU)                (None, 56, 56, 64)   0           batch_normalization_104[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_92 (Conv2D)              (None, 56, 56, 256)  16640       re_lu_111[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_93 (Conv2D)              (None, 56, 56, 256)  16640       max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_105 (BatchN (None, 56, 56, 256)  1024        conv2d_92[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_106 (BatchN (None, 56, 56, 256)  1024        conv2d_93[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_18 (Add)                    (None, 56, 56, 256)  0           batch_normalization_105[0][0]    \n",
      "                                                                 batch_normalization_106[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_112 (ReLU)                (None, 56, 56, 256)  0           add_18[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_94 (Conv2D)              (None, 56, 56, 64)   16448       re_lu_112[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_107 (BatchN (None, 56, 56, 64)   256         conv2d_94[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_113 (ReLU)                (None, 56, 56, 64)   0           batch_normalization_107[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_95 (Conv2D)              (None, 56, 56, 64)   36928       re_lu_113[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_108 (BatchN (None, 56, 56, 64)   256         conv2d_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_114 (ReLU)                (None, 56, 56, 64)   0           batch_normalization_108[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_96 (Conv2D)              (None, 56, 56, 256)  16640       re_lu_114[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_19 (Add)                    (None, 56, 56, 256)  0           conv2d_96[0][0]                  \n",
      "                                                                 re_lu_112[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_115 (ReLU)                (None, 56, 56, 256)  0           add_19[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_97 (Conv2D)              (None, 56, 56, 64)   16448       re_lu_115[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_109 (BatchN (None, 56, 56, 64)   256         conv2d_97[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_116 (ReLU)                (None, 56, 56, 64)   0           batch_normalization_109[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_98 (Conv2D)              (None, 56, 56, 64)   36928       re_lu_116[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_110 (BatchN (None, 56, 56, 64)   256         conv2d_98[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_117 (ReLU)                (None, 56, 56, 64)   0           batch_normalization_110[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_99 (Conv2D)              (None, 56, 56, 256)  16640       re_lu_117[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_20 (Add)                    (None, 56, 56, 256)  0           conv2d_99[0][0]                  \n",
      "                                                                 re_lu_115[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_118 (ReLU)                (None, 56, 56, 256)  0           add_20[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_100 (Conv2D)             (None, 28, 28, 128)  32896       re_lu_118[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_111 (BatchN (None, 28, 28, 128)  512         conv2d_100[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_119 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_111[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_101 (Conv2D)             (None, 28, 28, 128)  147584      re_lu_119[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_112 (BatchN (None, 28, 28, 128)  512         conv2d_101[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_120 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_112[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_102 (Conv2D)             (None, 28, 28, 512)  66048       re_lu_120[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_103 (Conv2D)             (None, 28, 28, 512)  131584      re_lu_118[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_113 (BatchN (None, 28, 28, 512)  2048        conv2d_102[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_114 (BatchN (None, 28, 28, 512)  2048        conv2d_103[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_21 (Add)                    (None, 28, 28, 512)  0           batch_normalization_113[0][0]    \n",
      "                                                                 batch_normalization_114[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_121 (ReLU)                (None, 28, 28, 512)  0           add_21[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_104 (Conv2D)             (None, 28, 28, 128)  65664       re_lu_121[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_115 (BatchN (None, 28, 28, 128)  512         conv2d_104[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_122 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_115[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_105 (Conv2D)             (None, 28, 28, 128)  147584      re_lu_122[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_116 (BatchN (None, 28, 28, 128)  512         conv2d_105[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_123 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_116[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_106 (Conv2D)             (None, 28, 28, 512)  66048       re_lu_123[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_22 (Add)                    (None, 28, 28, 512)  0           conv2d_106[0][0]                 \n",
      "                                                                 re_lu_121[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_124 (ReLU)                (None, 28, 28, 512)  0           add_22[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_107 (Conv2D)             (None, 28, 28, 128)  65664       re_lu_124[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_117 (BatchN (None, 28, 28, 128)  512         conv2d_107[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_125 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_117[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_108 (Conv2D)             (None, 28, 28, 128)  147584      re_lu_125[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_118 (BatchN (None, 28, 28, 128)  512         conv2d_108[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_126 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_118[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_109 (Conv2D)             (None, 28, 28, 512)  66048       re_lu_126[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_23 (Add)                    (None, 28, 28, 512)  0           conv2d_109[0][0]                 \n",
      "                                                                 re_lu_124[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_127 (ReLU)                (None, 28, 28, 512)  0           add_23[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_110 (Conv2D)             (None, 28, 28, 128)  65664       re_lu_127[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_119 (BatchN (None, 28, 28, 128)  512         conv2d_110[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_128 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_119[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_111 (Conv2D)             (None, 28, 28, 128)  147584      re_lu_128[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_120 (BatchN (None, 28, 28, 128)  512         conv2d_111[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_129 (ReLU)                (None, 28, 28, 128)  0           batch_normalization_120[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_112 (Conv2D)             (None, 28, 28, 512)  66048       re_lu_129[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_24 (Add)                    (None, 28, 28, 512)  0           conv2d_112[0][0]                 \n",
      "                                                                 re_lu_127[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_130 (ReLU)                (None, 28, 28, 512)  0           add_24[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_113 (Conv2D)             (None, 14, 14, 256)  131328      re_lu_130[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_121 (BatchN (None, 14, 14, 256)  1024        conv2d_113[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_131 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_121[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_114 (Conv2D)             (None, 14, 14, 256)  590080      re_lu_131[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_122 (BatchN (None, 14, 14, 256)  1024        conv2d_114[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_132 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_122[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_115 (Conv2D)             (None, 14, 14, 1024) 263168      re_lu_132[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_116 (Conv2D)             (None, 14, 14, 1024) 525312      re_lu_130[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_123 (BatchN (None, 14, 14, 1024) 4096        conv2d_115[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_124 (BatchN (None, 14, 14, 1024) 4096        conv2d_116[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_25 (Add)                    (None, 14, 14, 1024) 0           batch_normalization_123[0][0]    \n",
      "                                                                 batch_normalization_124[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_133 (ReLU)                (None, 14, 14, 1024) 0           add_25[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_117 (Conv2D)             (None, 14, 14, 256)  262400      re_lu_133[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_125 (BatchN (None, 14, 14, 256)  1024        conv2d_117[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_134 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_125[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_118 (Conv2D)             (None, 14, 14, 256)  590080      re_lu_134[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_126 (BatchN (None, 14, 14, 256)  1024        conv2d_118[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_135 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_126[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_119 (Conv2D)             (None, 14, 14, 1024) 263168      re_lu_135[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_26 (Add)                    (None, 14, 14, 1024) 0           conv2d_119[0][0]                 \n",
      "                                                                 re_lu_133[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_136 (ReLU)                (None, 14, 14, 1024) 0           add_26[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_120 (Conv2D)             (None, 14, 14, 256)  262400      re_lu_136[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_127 (BatchN (None, 14, 14, 256)  1024        conv2d_120[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_137 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_127[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_121 (Conv2D)             (None, 14, 14, 256)  590080      re_lu_137[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_128 (BatchN (None, 14, 14, 256)  1024        conv2d_121[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_138 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_128[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_122 (Conv2D)             (None, 14, 14, 1024) 263168      re_lu_138[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_27 (Add)                    (None, 14, 14, 1024) 0           conv2d_122[0][0]                 \n",
      "                                                                 re_lu_136[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_139 (ReLU)                (None, 14, 14, 1024) 0           add_27[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_123 (Conv2D)             (None, 14, 14, 256)  262400      re_lu_139[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_129 (BatchN (None, 14, 14, 256)  1024        conv2d_123[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_140 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_129[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_124 (Conv2D)             (None, 14, 14, 256)  590080      re_lu_140[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_130 (BatchN (None, 14, 14, 256)  1024        conv2d_124[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_141 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_130[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_125 (Conv2D)             (None, 14, 14, 1024) 263168      re_lu_141[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_28 (Add)                    (None, 14, 14, 1024) 0           conv2d_125[0][0]                 \n",
      "                                                                 re_lu_139[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_142 (ReLU)                (None, 14, 14, 1024) 0           add_28[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_126 (Conv2D)             (None, 14, 14, 256)  262400      re_lu_142[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_131 (BatchN (None, 14, 14, 256)  1024        conv2d_126[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_143 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_131[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_127 (Conv2D)             (None, 14, 14, 256)  590080      re_lu_143[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_132 (BatchN (None, 14, 14, 256)  1024        conv2d_127[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_144 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_132[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_128 (Conv2D)             (None, 14, 14, 1024) 263168      re_lu_144[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_29 (Add)                    (None, 14, 14, 1024) 0           conv2d_128[0][0]                 \n",
      "                                                                 re_lu_142[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_145 (ReLU)                (None, 14, 14, 1024) 0           add_29[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_129 (Conv2D)             (None, 14, 14, 256)  262400      re_lu_145[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_133 (BatchN (None, 14, 14, 256)  1024        conv2d_129[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_146 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_133[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_130 (Conv2D)             (None, 14, 14, 256)  590080      re_lu_146[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_134 (BatchN (None, 14, 14, 256)  1024        conv2d_130[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_147 (ReLU)                (None, 14, 14, 256)  0           batch_normalization_134[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_131 (Conv2D)             (None, 14, 14, 1024) 263168      re_lu_147[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_30 (Add)                    (None, 14, 14, 1024) 0           conv2d_131[0][0]                 \n",
      "                                                                 re_lu_145[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_148 (ReLU)                (None, 14, 14, 1024) 0           add_30[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_132 (Conv2D)             (None, 7, 7, 512)    524800      re_lu_148[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_135 (BatchN (None, 7, 7, 512)    2048        conv2d_132[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_149 (ReLU)                (None, 7, 7, 512)    0           batch_normalization_135[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_133 (Conv2D)             (None, 7, 7, 512)    2359808     re_lu_149[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_136 (BatchN (None, 7, 7, 512)    2048        conv2d_133[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_150 (ReLU)                (None, 7, 7, 512)    0           batch_normalization_136[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_134 (Conv2D)             (None, 7, 7, 2048)   1050624     re_lu_150[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_135 (Conv2D)             (None, 7, 7, 2048)   2099200     re_lu_148[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_137 (BatchN (None, 7, 7, 2048)   8192        conv2d_134[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_138 (BatchN (None, 7, 7, 2048)   8192        conv2d_135[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "add_31 (Add)                    (None, 7, 7, 2048)   0           batch_normalization_137[0][0]    \n",
      "                                                                 batch_normalization_138[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_151 (ReLU)                (None, 7, 7, 2048)   0           add_31[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_136 (Conv2D)             (None, 7, 7, 512)    1049088     re_lu_151[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_139 (BatchN (None, 7, 7, 512)    2048        conv2d_136[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_152 (ReLU)                (None, 7, 7, 512)    0           batch_normalization_139[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_137 (Conv2D)             (None, 7, 7, 512)    2359808     re_lu_152[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_140 (BatchN (None, 7, 7, 512)    2048        conv2d_137[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_153 (ReLU)                (None, 7, 7, 512)    0           batch_normalization_140[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_138 (Conv2D)             (None, 7, 7, 2048)   1050624     re_lu_153[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_32 (Add)                    (None, 7, 7, 2048)   0           conv2d_138[0][0]                 \n",
      "                                                                 re_lu_151[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_154 (ReLU)                (None, 7, 7, 2048)   0           add_32[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_139 (Conv2D)             (None, 7, 7, 512)    1049088     re_lu_154[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_141 (BatchN (None, 7, 7, 512)    2048        conv2d_139[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_155 (ReLU)                (None, 7, 7, 512)    0           batch_normalization_141[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_140 (Conv2D)             (None, 7, 7, 512)    2359808     re_lu_155[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_142 (BatchN (None, 7, 7, 512)    2048        conv2d_140[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_156 (ReLU)                (None, 7, 7, 512)    0           batch_normalization_142[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_141 (Conv2D)             (None, 7, 7, 2048)   1050624     re_lu_156[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_33 (Add)                    (None, 7, 7, 2048)   0           conv2d_141[0][0]                 \n",
      "                                                                 re_lu_154[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_157 (ReLU)                (None, 7, 7, 2048)   0           add_33[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_1 (Glo (None, 2048)         0           re_lu_157[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1000)         2049000     global_average_pooling2d_1[0][0] \n",
      "==================================================================================================\n",
      "Total params: 25,591,656\n",
      "Trainable params: 25,561,064\n",
      "Non-trainable params: 30,592\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.layers import Input, Conv2D, BatchNormalization, ReLU, Add, MaxPool2D, GlobalAvgPool2D, Dense\n",
    "\n",
    "\n",
    "def conv_batchnorm_relu(x, filters, kernel_size, strides):\n",
    "    x = Conv2D(filters=filters,\n",
    "               kernel_size=kernel_size,\n",
    "               strides=strides,\n",
    "               padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = ReLU()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def identity_block(tensor, filters):\n",
    "    x = conv_batchnorm_relu(tensor, filters=filters, kernel_size=1, strides=1)\n",
    "    x = conv_batchnorm_relu(x, filters=filters, kernel_size=3, strides=1)\n",
    "    x = Conv2D(filters=4 * filters, kernel_size=1, strides=1)(x)\n",
    "    x = Add()([x, tensor])\n",
    "    x = ReLU()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def projection_block(tensor, filters, strides):\n",
    "    x = conv_batchnorm_relu(tensor,\n",
    "                            filters=filters,\n",
    "                            kernel_size=1,\n",
    "                            strides=strides)\n",
    "    x = conv_batchnorm_relu(x, filters=filters, kernel_size=3, strides=1)\n",
    "    x = Conv2D(filters=4 * filters, kernel_size=1,\n",
    "               strides=1)(x)  # notice: filters=4*filters\n",
    "    x = BatchNormalization()(x)\n",
    "\n",
    "    # right stream\n",
    "    shortcut = Conv2D(filters=4 * filters, kernel_size=1,\n",
    "                      strides=strides)(tensor)  # notice: filters=4*filters\n",
    "    shortcut = BatchNormalization()(shortcut)\n",
    "\n",
    "    x = Add()([x, shortcut])\n",
    "    x = ReLU()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def resnet_block(x, filters, reps, strides):\n",
    "    x = projection_block(x, filters=filters, strides=strides)\n",
    "    for _ in range(reps -\n",
    "                   1):  # the -1 is because the first block was a Conv one\n",
    "        x = identity_block(x, filters=filters)\n",
    "    return x\n",
    "\n",
    "\n",
    "input = Input(shape=(224, 224, 3))\n",
    "\n",
    "x = conv_batchnorm_relu(input, filters=64, kernel_size=7,\n",
    "                        strides=2)  # [3]: 7x7, 64, strides 2\n",
    "x = MaxPool2D(pool_size=3, strides=2,\n",
    "              padding='same')(x)  # [3]: 3x3 max mool, strides 2\n",
    "\n",
    "x = resnet_block(x, filters=64, reps=3, strides=1)\n",
    "x = resnet_block(x, filters=128, reps=4, strides=2)  # s=2 ([2]: conv3_1)\n",
    "x = resnet_block(x, filters=256, reps=6, strides=2)  # s=2 ([2]: conv4_1)\n",
    "x = resnet_block(x, filters=512, reps=3, strides=2)  # s=2 ([2]: conv5_1)\n",
    "\n",
    "x = GlobalAvgPool2D()(\n",
    "    x)  # [3]: average pool *it is not written any pool size so we use Global\n",
    "\n",
    "output = Dense(1000, activation='softmax')(x)  # [3]: 1000-d fc, softmax\n",
    "\n",
    "from tensorflow.keras import Model\n",
    "\n",
    "model = Model(input, output)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Xception"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T08:00:19.438716Z",
     "start_time": "2020-04-09T08:00:18.124490Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Conv2D, SeparableConv2D, Add, Dense, BatchNormalization, ReLU, MaxPool2D, GlobalAvgPool2D\n",
    "\n",
    "\n",
    "def conv_bn(x, filters, kernel_size, strides=1):\n",
    "    x = Conv2D(filters=filters,\n",
    "               kernel_size=kernel_size,\n",
    "               strides=strides,\n",
    "               padding='same',\n",
    "               use_bias=False)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def sep_bn(x, filters, kernel_size, strides=1):\n",
    "    x = SeparableConv2D(filters=filters,\n",
    "                        kernel_size=kernel_size,\n",
    "                        strides=strides,\n",
    "                        padding='same',\n",
    "                        use_bias=False)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def entry_flow(x):\n",
    "    x = conv_bn(x, filters=32, kernel_size=3, strides=2)\n",
    "    x = ReLU()(x)\n",
    "    x = conv_bn(x, filters=64, kernel_size=3)\n",
    "    tensor = ReLU()(x)\n",
    "\n",
    "    x = sep_bn(tensor, filters=128, kernel_size=3)\n",
    "    x = ReLU()(x)\n",
    "    x = sep_bn(x, filters=128, kernel_size=3)\n",
    "    x = MaxPool2D(pool_size=3, strides=2, padding='same')(x)\n",
    "\n",
    "    tensor = conv_bn(tensor, filters=128, kernel_size=1, strides=2)\n",
    "\n",
    "    x = Add()([tensor, x])\n",
    "    x = ReLU()(x)\n",
    "    x = sep_bn(x, filters=256, kernel_size=3)\n",
    "    x = ReLU()(x)\n",
    "    x = sep_bn(x, filters=256, kernel_size=3)\n",
    "    x = MaxPool2D(pool_size=3, strides=2, padding='same')(x)\n",
    "\n",
    "    tensor = conv_bn(tensor, filters=256, kernel_size=1, strides=2)\n",
    "\n",
    "    x = Add()([tensor, x])\n",
    "    x = ReLU()(x)\n",
    "    x = sep_bn(x, filters=728, kernel_size=3)\n",
    "    x = ReLU()(x)\n",
    "    x = sep_bn(x, filters=728, kernel_size=3)\n",
    "    x = MaxPool2D(pool_size=3, strides=2, padding='same')(x)\n",
    "\n",
    "    tensor = conv_bn(tensor, filters=728, kernel_size=1, strides=2)\n",
    "    x = Add()([tensor, x])\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "def middle_flow(tensor):\n",
    "    for _ in range(8):\n",
    "        x = ReLU()(tensor)\n",
    "        x = sep_bn(x, filters=728, kernel_size=3)\n",
    "        x = ReLU()(x)\n",
    "        x = sep_bn(x, filters=728, kernel_size=3)\n",
    "        x = ReLU()(x)\n",
    "        x = sep_bn(x, filters=728, kernel_size=3)\n",
    "\n",
    "        tensor = Add()([tensor, x])\n",
    "\n",
    "    return tensor\n",
    "\n",
    "\n",
    "def exit_flow(tensor):\n",
    "    x = ReLU()(tensor)\n",
    "    x = sep_bn(x, filters=728, kernel_size=3)\n",
    "    x = ReLU()(x)\n",
    "    x = sep_bn(x, filters=1024, kernel_size=3)\n",
    "    x = MaxPool2D(3, strides=2, padding='same')(x)\n",
    "\n",
    "    tensor = conv_bn(tensor, filters=1024, kernel_size=1, strides=2)\n",
    "\n",
    "    x = Add()([tensor, x])\n",
    "    x = sep_bn(x, filters=1536, kernel_size=3)\n",
    "    x = ReLU()(x)\n",
    "    x = sep_bn(x, filters=2048, kernel_size=3)\n",
    "    x = ReLU()(x)\n",
    "    x = GlobalAvgPool2D()(x)\n",
    "    x = Dense(units=1000, activation='softmax')(x)\n",
    "\n",
    "    return x\n",
    "\n",
    "\n",
    "input = Input(shape=[299, 299, 3])\n",
    "\n",
    "x = entry_flow(input)\n",
    "x = middle_flow(x)\n",
    "output = exit_flow(x)\n",
    "\n",
    "from tensorflow.keras import Model\n",
    "model = Model(input, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-09T08:00:25.220028Z",
     "start_time": "2020-04-09T08:00:25.190206Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_3\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_6 (InputLayer)            [(None, 299, 299, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_142 (Conv2D)             (None, 150, 150, 32) 864         input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_143 (BatchN (None, 150, 150, 32) 128         conv2d_142[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_158 (ReLU)                (None, 150, 150, 32) 0           batch_normalization_143[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_143 (Conv2D)             (None, 150, 150, 64) 18432       re_lu_158[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_144 (BatchN (None, 150, 150, 64) 256         conv2d_143[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_159 (ReLU)                (None, 150, 150, 64) 0           batch_normalization_144[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d (SeparableConv (None, 150, 150, 128 8768        re_lu_159[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_145 (BatchN (None, 150, 150, 128 512         separable_conv2d[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_160 (ReLU)                (None, 150, 150, 128 0           batch_normalization_145[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_1 (SeparableCo (None, 150, 150, 128 17536       re_lu_160[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_144 (Conv2D)             (None, 75, 75, 128)  8192        re_lu_159[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_146 (BatchN (None, 150, 150, 128 512         separable_conv2d_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_147 (BatchN (None, 75, 75, 128)  512         conv2d_144[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 75, 75, 128)  0           batch_normalization_146[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "add_34 (Add)                    (None, 75, 75, 128)  0           batch_normalization_147[0][0]    \n",
      "                                                                 max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_161 (ReLU)                (None, 75, 75, 128)  0           add_34[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_2 (SeparableCo (None, 75, 75, 256)  33920       re_lu_161[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_148 (BatchN (None, 75, 75, 256)  1024        separable_conv2d_2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_162 (ReLU)                (None, 75, 75, 256)  0           batch_normalization_148[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_3 (SeparableCo (None, 75, 75, 256)  67840       re_lu_162[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_145 (Conv2D)             (None, 38, 38, 256)  32768       batch_normalization_147[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_149 (BatchN (None, 75, 75, 256)  1024        separable_conv2d_3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_150 (BatchN (None, 38, 38, 256)  1024        conv2d_145[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)  (None, 38, 38, 256)  0           batch_normalization_149[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "add_35 (Add)                    (None, 38, 38, 256)  0           batch_normalization_150[0][0]    \n",
      "                                                                 max_pooling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_163 (ReLU)                (None, 38, 38, 256)  0           add_35[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_4 (SeparableCo (None, 38, 38, 728)  188672      re_lu_163[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_151 (BatchN (None, 38, 38, 728)  2912        separable_conv2d_4[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_164 (ReLU)                (None, 38, 38, 728)  0           batch_normalization_151[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_5 (SeparableCo (None, 38, 38, 728)  536536      re_lu_164[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_146 (Conv2D)             (None, 19, 19, 728)  186368      batch_normalization_150[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_152 (BatchN (None, 38, 38, 728)  2912        separable_conv2d_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_153 (BatchN (None, 19, 19, 728)  2912        conv2d_146[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)  (None, 19, 19, 728)  0           batch_normalization_152[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "add_36 (Add)                    (None, 19, 19, 728)  0           batch_normalization_153[0][0]    \n",
      "                                                                 max_pooling2d_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_165 (ReLU)                (None, 19, 19, 728)  0           add_36[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_6 (SeparableCo (None, 19, 19, 728)  536536      re_lu_165[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_154 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_166 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_154[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_7 (SeparableCo (None, 19, 19, 728)  536536      re_lu_166[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_155 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_7[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_167 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_155[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_8 (SeparableCo (None, 19, 19, 728)  536536      re_lu_167[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_156 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_8[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "add_37 (Add)                    (None, 19, 19, 728)  0           add_36[0][0]                     \n",
      "                                                                 batch_normalization_156[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_168 (ReLU)                (None, 19, 19, 728)  0           add_37[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_9 (SeparableCo (None, 19, 19, 728)  536536      re_lu_168[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_157 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_9[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_169 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_157[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_10 (SeparableC (None, 19, 19, 728)  536536      re_lu_169[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_158 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_10[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_170 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_158[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_11 (SeparableC (None, 19, 19, 728)  536536      re_lu_170[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_159 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_11[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_38 (Add)                    (None, 19, 19, 728)  0           add_37[0][0]                     \n",
      "                                                                 batch_normalization_159[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_171 (ReLU)                (None, 19, 19, 728)  0           add_38[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_12 (SeparableC (None, 19, 19, 728)  536536      re_lu_171[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_160 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_12[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_172 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_160[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_13 (SeparableC (None, 19, 19, 728)  536536      re_lu_172[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_161 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_13[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_173 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_161[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_14 (SeparableC (None, 19, 19, 728)  536536      re_lu_173[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_162 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_39 (Add)                    (None, 19, 19, 728)  0           add_38[0][0]                     \n",
      "                                                                 batch_normalization_162[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_174 (ReLU)                (None, 19, 19, 728)  0           add_39[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_15 (SeparableC (None, 19, 19, 728)  536536      re_lu_174[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_163 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_15[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_175 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_163[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_16 (SeparableC (None, 19, 19, 728)  536536      re_lu_175[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_164 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_16[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_176 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_164[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_17 (SeparableC (None, 19, 19, 728)  536536      re_lu_176[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_165 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_17[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_40 (Add)                    (None, 19, 19, 728)  0           add_39[0][0]                     \n",
      "                                                                 batch_normalization_165[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_177 (ReLU)                (None, 19, 19, 728)  0           add_40[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_18 (SeparableC (None, 19, 19, 728)  536536      re_lu_177[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_166 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_18[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_178 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_166[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_19 (SeparableC (None, 19, 19, 728)  536536      re_lu_178[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_167 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_19[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_179 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_167[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_20 (SeparableC (None, 19, 19, 728)  536536      re_lu_179[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_168 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_20[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_41 (Add)                    (None, 19, 19, 728)  0           add_40[0][0]                     \n",
      "                                                                 batch_normalization_168[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_180 (ReLU)                (None, 19, 19, 728)  0           add_41[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_21 (SeparableC (None, 19, 19, 728)  536536      re_lu_180[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_169 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_21[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_181 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_169[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_22 (SeparableC (None, 19, 19, 728)  536536      re_lu_181[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_170 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_22[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_182 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_170[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_23 (SeparableC (None, 19, 19, 728)  536536      re_lu_182[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_171 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_23[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_42 (Add)                    (None, 19, 19, 728)  0           add_41[0][0]                     \n",
      "                                                                 batch_normalization_171[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_183 (ReLU)                (None, 19, 19, 728)  0           add_42[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_24 (SeparableC (None, 19, 19, 728)  536536      re_lu_183[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_172 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_24[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_184 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_172[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_25 (SeparableC (None, 19, 19, 728)  536536      re_lu_184[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_173 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_25[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_185 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_173[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_26 (SeparableC (None, 19, 19, 728)  536536      re_lu_185[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_174 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_26[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_43 (Add)                    (None, 19, 19, 728)  0           add_42[0][0]                     \n",
      "                                                                 batch_normalization_174[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_186 (ReLU)                (None, 19, 19, 728)  0           add_43[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_27 (SeparableC (None, 19, 19, 728)  536536      re_lu_186[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_175 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_27[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_187 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_175[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_28 (SeparableC (None, 19, 19, 728)  536536      re_lu_187[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_176 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_28[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_188 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_176[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_29 (SeparableC (None, 19, 19, 728)  536536      re_lu_188[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_177 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_29[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "add_44 (Add)                    (None, 19, 19, 728)  0           add_43[0][0]                     \n",
      "                                                                 batch_normalization_177[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_189 (ReLU)                (None, 19, 19, 728)  0           add_44[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_30 (SeparableC (None, 19, 19, 728)  536536      re_lu_189[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_178 (BatchN (None, 19, 19, 728)  2912        separable_conv2d_30[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_190 (ReLU)                (None, 19, 19, 728)  0           batch_normalization_178[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_31 (SeparableC (None, 19, 19, 1024) 752024      re_lu_190[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_147 (Conv2D)             (None, 10, 10, 1024) 745472      add_44[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_179 (BatchN (None, 19, 19, 1024) 4096        separable_conv2d_31[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_180 (BatchN (None, 10, 10, 1024) 4096        conv2d_147[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2D)  (None, 10, 10, 1024) 0           batch_normalization_179[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "add_45 (Add)                    (None, 10, 10, 1024) 0           batch_normalization_180[0][0]    \n",
      "                                                                 max_pooling2d_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_32 (SeparableC (None, 10, 10, 1536) 1582080     add_45[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_181 (BatchN (None, 10, 10, 1536) 6144        separable_conv2d_32[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_191 (ReLU)                (None, 10, 10, 1536) 0           batch_normalization_181[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "separable_conv2d_33 (SeparableC (None, 10, 10, 2048) 3159552     re_lu_191[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_182 (BatchN (None, 10, 10, 2048) 8192        separable_conv2d_33[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "re_lu_192 (ReLU)                (None, 10, 10, 2048) 0           batch_normalization_182[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_2 (Glo (None, 2048)         0           re_lu_192[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 1000)         2049000     global_average_pooling2d_2[0][0] \n",
      "==================================================================================================\n",
      "Total params: 22,910,480\n",
      "Trainable params: 22,855,952\n",
      "Non-trainable params: 54,528\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "631.85px",
    "left": "705px",
    "right": "20px",
    "top": "120px",
    "width": "357px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
